package com.eric.r2d2.urlbase;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;

public class UrlBasePoll {
	// 设定每个文件取多少条
	static int n = 50;

	public static void main(String[] args) throws IOException {
		long a = System.currentTimeMillis();
		if (args.length < 2) {
			System.out.println("Usage:Poll file_in file_out [Number to retrieve]");
			System.exit(1);
		}
		if (args.length == 3)
			n = Integer.parseInt(args[2]);

		//定义文件读写
		File file_in = new File(args[0]);
		File file_out = new File(args[1]);
		BufferedReader br = new BufferedReader(new FileReader(file_in));
		BufferedWriter bw_out = new BufferedWriter(new FileWriter(file_out, true));

/*新建3个队列
 * queue_in用于保存原url_base队列
 * tmp_queue用于保存待poll队列，进行后续处理
 * queue_out用于保存处理后的poll队列
*/
		BlockingQueue<JSONObject> queue_in = new LinkedBlockingQueue<JSONObject>();
		BlockingQueue<JSONObject> tmp_queue = new LinkedBlockingQueue<JSONObject>();
		BlockingQueue<JSONObject> queue_out = new LinkedBlockingQueue<JSONObject>();
		
		//Priority Map用于暂存url_base取出的url，根据其priority值，保存到对应的queue
		Map<Integer,BlockingQueue<JSONObject>> priority_map = new HashMap<Integer,BlockingQueue<JSONObject>>();
		for (int i =1;i<=10;i++) priority_map.put(i, new LinkedBlockingQueue<JSONObject>());

		// 将目标url小文件读取到priority map里
		String line;
		while ((line = br.readLine()) != null) {
			priority_map.get(JSON.parseObject(line).getInteger("priority")).add(JSON.parseObject(line));
		}
		br.close();

//		// 保证n不超过queue的大小
//		if (n > queue.size())
//			n = queue.size();
//		 Poll queue，并保存到临时队列
//		for (int i = 0; i < n; i++) {
//			tmp.add(queue.poll());
//		}
		//依次对priority map的队列进行取值，并保存进tmp queue，优先级1取1个，优先级2取2个，以此类推
		for (int i =1;i<=10;i++) {
			//tmp队列的大小超过需要poll的数量n时，跳出循环，不再取
			if (tmp_queue.size() > n) break;
			
			//保证n不超过queue大小，否则回返回null pointer
			int size = priority_map.get(i).size()>n ? n : priority_map.get(i).size();
			
			for (int j = size;j>0;j--) {
				//tmp队列的大小超过需要poll的数量n时，跳出循环，不再取
				if (tmp_queue.size() > n) break;
				
				tmp_queue.add(priority_map.get(i).poll());
			}
		}
		
		//剩下的priority map队列保存进queue
		for (int i = 1;i<=10;i++) {
			for (JSONObject j : priority_map.get(i)) queue_in.add(j);
		}

		// 修改临时队列tmp_queue后，分别加到原queue_in，以及queue_out
		for (JSONObject j : tmp_queue) {
//			String s = j.toJSONString()
//修改crawl_date
//					.replaceAll(j.getString("crawl_date"), new SimpleDateFormat("yyyyMMdd").format(new Date()))
//					.replaceAll(j.getString("crawled"), "yes");
			JSONObject inner = new JSONObject();
			inner.put("$date", new Date());
			j.put("crawl_date",inner);
			
			j.put("crawled", "yes");
			
			queue_in.add(j);
			queue_out.add(j);
		}
		// 保存queue_out到文件
		for (JSONObject j : queue_out) {
			bw_out.append(j.toJSONString());
			bw_out.append("\n");
			bw_out.flush();
		}
		bw_out.close();

		// 更新原queue文件
		file_in.delete();
		BufferedWriter bw_in = new BufferedWriter(new FileWriter(file_in, true));
		for (JSONObject j : queue_in) {
			bw_in.append(j.toJSONString());
			bw_in.append("\n");
			bw_in.flush();
		}
		bw_in.close();
		System.out.println(System.currentTimeMillis() - a);
	}
}
